package page;

import java.io.IOException;
import java.util.Collection;
import java.util.HashSet;
import java.util.NoSuchElementException;
import java.util.Set;
import java.util.concurrent.LinkedBlockingQueue;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;

/**
 * PageRetriever class retrieves web pages and stores
 * them for later analysis by the PageParser class. 
 * 
 * @version February 6, 2012
 * @author Jay Fernandez
 * @author Kyle Richardson
 */
public class PageRetriever {
	/** Queue of web pages waiting to be parsed. */
	LinkedBlockingQueue<String> waiting;

	/** Set of URLs already visited. */
	Set<String> visited;
	
	/**
	 * Constructor that initializes with a starting address.
	 * 
	 * @param starting_address The first address.
	 * @throws IOException if not acceptable address.
	 */
	public PageRetriever(String starting_address) throws IOException {
		waiting = new LinkedBlockingQueue<String>();
		visited = new HashSet<String>();
		
		waiting.add(starting_address);
	}
	
	/**
	 * Adds a collection of unvisited web pages.
	 * 
	 * @param addresses The collection of addresses.
	 */
	public void addAllPages(Collection<? extends String> addresses) {
		for (String address : addresses) {
			waiting.add(address);
		}
	}
	
	/**
	 * @return the next available web page's content.
	 * @throws NoSuchElementException if queue is empty
	 */
	public Document getNextPage() {
		Document doc = null;
		boolean validURL = false;
		String next = waiting.remove();
		while(visited.contains(next)) {
			next = waiting.remove();
		}
		
		while (!validURL) {
			try {
				 doc = Jsoup.connect(next).get();
				 validURL = true;
			} catch (IOException e) {
				next = waiting.remove();
			} catch (IllegalArgumentException e) {
				next = waiting.remove();
			} catch (NullPointerException e) {
				next = waiting.remove();
			}
		}
		visited.add(next);
		return doc;
	}
}
